rm(list = ls())

## ##########
## init

## Load libraries

library(foreign, warn.conflicts = FALSE)
library(data.table, warn.conflicts = FALSE)
suppressPackageStartupMessages(library(tidyverse, warn.conflicts = FALSE))

## Load data

bes <- read.spss("../source_data/BES2019_W21_Panel_v21.0.sav", to.data.frame = TRUE)

pacer <- read.spss("../source_data/P_UCL_PACERSurvey_April2020_CLIENT.sav", to.data.frame = TRUE)

pacer_w2 <- read.spss("../source_data/P_UCL_PACERSurvey_September2020_CLIENT.sav", to.data.frame = TRUE)

# Drop BES ids who appear in zero waves

bes <- bes[!apply(bes[,paste0("wave",1:21)],1, function(x) all(is.na(x))),]

## ##########
## Reconcile BES and PACER factor levels

# RedistSelf

pacer$redistSelf <- as.character(pacer$redistSelf)
pacer$redistSelf[pacer$redistSelf == "Government should be less concerned about equal incomes  10"] <- "Government should be less concerned about equal incomes"
pacer$redistSelf[pacer$redistSelf == "Government should try to make incomes equal  0"] <- "Government should try to make incomes equal"

pacer$redistSelf <- factor(pacer$redistSelf, levels = levels(bes$redistSelfW1))

pacer_w2$redistSelf <- as.character(pacer_w2$redistSelf)
pacer_w2$redistSelf[pacer_w2$redistSelf == "Government should be less concerned about equal incomes  10"] <- "Government should be less concerned about equal incomes"
pacer_w2$redistSelf[pacer_w2$redistSelf == "Government should try to make incomes equal  0"] <- "Government should try to make incomes equal"
pacer_w2$redistSelf <- factor(pacer_w2$redistSelf, levels = levels(bes$redistSelfW1))

# ptvCon

pacer$ptvCon <- as.character(pacer$ptvCon)
pacer$ptvCon[pacer$ptvCon == "Very unlikely  0"] <- "Very unlikely"
pacer$ptvCon[pacer$ptvCon == "Very likely  10"] <- "Very likely"
pacer$ptvCon <- factor(pacer$ptvCon, levels = levels(bes$ptvConW1))

pacer_w2$ptvCon <- as.character(pacer_w2$ptvCon)
pacer_w2$ptvCon[pacer_w2$ptvCon == "Very unlikely  0"] <- "Very unlikely"
pacer_w2$ptvCon[pacer_w2$ptvCon == "Very likely  10"] <- "Very likely"
pacer_w2$ptvCon <- factor(pacer_w2$ptvCon, levels = levels(bes$ptvConW1))

# ptvLab

pacer$ptvLab <- as.character(pacer$ptvLab)
pacer$ptvLab[pacer$ptvLab == "Very unlikely  0"] <- "Very unlikely"
pacer$ptvLab[pacer$ptvLab == "Very likely  10"] <- "Very likely"
pacer$ptvLab <- factor(pacer$ptvLab, levels = levels(bes$ptvLabW1))

pacer_w2$ptvLab <- as.character(pacer_w2$ptvLab)
pacer_w2$ptvLab[pacer_w2$ptvLab == "Very unlikely  0"] <- "Very unlikely"
pacer_w2$ptvLab[pacer_w2$ptvLab == "Very likely  10"] <- "Very likely"
pacer_w2$ptvLab <- factor(pacer_w2$ptvLab, levels = levels(bes$ptvLabW1))

# ptvLD

pacer$ptvLD <- as.character(pacer$ptvLD)
pacer$ptvLD[pacer$ptvLD == "Very unlikely  0"] <- "Very unlikely"
pacer$ptvLD[pacer$ptvLD == "Very likely  10"] <- "Very likely"
pacer$ptvLD <- factor(pacer$ptvLD, levels = levels(bes$ptvLDW1))

pacer_w2$ptvLD <- as.character(pacer_w2$ptvLD)
pacer_w2$ptvLD[pacer_w2$ptvLD == "Very unlikely  0"] <- "Very unlikely"
pacer_w2$ptvLD[pacer_w2$ptvLD == "Very likely  10"] <- "Very likely"
pacer_w2$ptvLD <- factor(pacer_w2$ptvLD, levels = levels(bes$ptvLDW1))

# taxSpendSelf

pacer$taxSpendSelf <- as.character(pacer$taxSpendSelf)
pacer$taxSpendSelf[pacer$taxSpendSelf == "Government should increase taxes a lot and spend much more on health and social services  10"] <- "Government should increase taxes a lot and spend much more on health and social services"
pacer$taxSpendSelf[pacer$taxSpendSelf == "Government should cut taxes a lot and spend much less on health and social services  0"] <- "Government should cut taxes a lot and spend much less on health and social services"
pacer$taxSpendSelf <- factor(pacer$taxSpendSelf, levels = levels(bes$taxSpendSelfW14))

pacer_w2$taxSpendSelf <- as.character(pacer_w2$taxSpendSelf)
pacer_w2$taxSpendSelf[pacer_w2$taxSpendSelf == "Government should increase taxes a lot and spend much more on health and social services  10"] <- "Government should increase taxes a lot and spend much more on health and social services"
pacer_w2$taxSpendSelf[pacer_w2$taxSpendSelf == "Government should cut taxes a lot and spend much less on health and social services  0"] <- "Government should cut taxes a lot and spend much less on health and social services"
pacer_w2$taxSpendSelf <- factor(pacer_w2$taxSpendSelf, levels = levels(bes$taxSpendSelfW14))

# trustWestminster
bes$trustWestminsterW19 <- as.character(bes$trustWestminsterW19)
bes$trustWestminsterW19 <- factor(bes$trustWestminsterW19, levels = levels(pacer$trustWestminster))

# deficitReduce

levels(pacer$deficitReduce)[3] <- "It is important but not absolutely  necessary"
pacer$deficitReduce <- factor(pacer$deficitReduce, levels = levels(bes$deficitReduceW10))

levels(pacer_w2$deficitReduce)[3] <- "It is important but not absolutely  necessary"
pacer_w2$deficitReduce <- factor(pacer_w2$deficitReduce, levels = levels(bes$deficitReduceW10))

# riskTaking

pacer$riskTaking <- factor(pacer$riskTaking, levels = levels(bes$riskTakingW1_W5))
pacer_w2$riskTaking <- factor(pacer_w2$riskTaking, levels = levels(bes$riskTakingW1_W5))

# approveUKGovt

pacer$approveUKGovt <- factor(pacer$approveUKGovt, levels = levels(bes$approveUKGovtW1))
pacer_w2$approveUKGovt <- factor(pacer_w2$approveUKGovt, levels = levels(bes$approveUKGovtW1))

# govtHandle...

pacer$govtHandleCostLive <- droplevels(pacer$govtHandleCostLive)
pacer_w2$govtHandleCostLive <- droplevels(pacer_w2$govtHandleCostLive)

pacer$govtHandleEcon <- droplevels(pacer$govtHandleEcon)
pacer_w2$govtHandleEcon <- droplevels(pacer_w2$govtHandleEcon)

pacer$govtHandleImmig <- droplevels(pacer$govtHandleImmig)
pacer_w2$govtHandleImmig <- droplevels(pacer_w2$govtHandleImmig)

pacer$govtHandleNHS <- droplevels(pacer$govtHandleNHS)
pacer_w2$govtHandleNHS <- droplevels(pacer_w2$govtHandleNHS)

pacer$govtHandleEduc <- droplevels(pacer$govtHandleEduc)
pacer_w2$govtHandleEduc <- droplevels(pacer_w2$govtHandleEduc)

pacer$govtHandleLevelCrime <- droplevels(pacer$govtHandleLevelCrime)
pacer_w2$govtHandleLevelCrime <- droplevels(pacer_w2$govtHandleLevelCrime)

# govtCorpSupport

pacer$govtCorpSupport <- droplevels(pacer$govtCorpSupport)
pacer_w2$govtCorpSupport <- droplevels(pacer_w2$govtCorpSupport)

bes$govtCorpSupportW20 <- droplevels(bes$govtCorpSupportW20)
bes$govtCorpSupportW20 <- as.character(bes$govtCorpSupportW20)
bes$govtCorpSupportW20[which(bes$govtCorpSupportW20 == "Don't know")] <- "Don’t know"
bes$govtCorpSupportW20 <- factor(bes$govtCorpSupportW20, levels = levels(pacer_w2$govtCorpSupport))

# ideoBHPS... 
pacer$ideoBHPSPprivateEnterprise <- droplevels(pacer$ideoBHPSPprivateEnterprise)
pacer_w2$ideoBHPSPprivateEnterprise <- droplevels(pacer_w2$ideoBHPSPprivateEnterprise)
levels(bes$privateEnterpriseW20)[levels(bes$privateEnterpriseW20) == "Don't know"] <- "Don’t know" 
bes$ideoBHPSPprivateEnterpriseW20 <- bes$privateEnterpriseW20
#levels(bes$privateEnterpriseW21)[levels(bes$privateEnterpriseW21) == "Don't know"] <- "Don’t know" 
#bes$ideoBHPSPprivateEnterpriseW21 <- bes$privateEnterpriseW21

pacer$ideoBHPSSstateOwnership <- droplevels(pacer$ideoBHPSSstateOwnership)
pacer_w2$ideoBHPSSstateOwnership <- droplevels(pacer_w2$ideoBHPSSstateOwnership)
levels(bes$stateOwnershipW20)[levels(bes$stateOwnershipW20) == "Don't know"] <- "Don’t know"  
bes$ideoBHPSSstateOwnershipW20 <- bes$stateOwnershipW20
#levels(bes$stateOwnershipW21)[levels(bes$stateOwnershipW21) == "Don't know"] <- "Don’t know"  
#bes$ideoBHPSSstateOwnershipW21 <- bes$stateOwnershipW21

pacer$ideoBHPSJjobForAll <- droplevels(pacer$ideoBHPSJjobForAll)
pacer_w2$ideoBHPSJjobForAll <- droplevels(pacer_w2$ideoBHPSJjobForAll)
levels(bes$jobForAllW20)[levels(bes$jobForAllW20) == "Don't know"] <- "Don’t know"  
bes$ideoBHPSJjobForAllW20 <- bes$jobForAllW20
#levels(bes$jobForAllW21)[levels(bes$jobForAllW21) == "Don't know"] <- "Don’t know"  
#bes$ideoBHPSJjobForAllW21 <- bes$jobForAllW21

pacer$ideoNoNeedUnions <- droplevels(pacer$ideoNoNeedUnions)
pacer_w2$ideoNoNeedUnions <- droplevels(pacer_w2$ideoNoNeedUnions)

# bes$lr3 and pacer$ideoFairShareWealth -- Needs to be reversed to be consistent between PACER and BES waves
## PACER wording: "Ordinary people get their fair share of the nation's wealth"
## BES wording: "Ordinary working people do not get their fair share of the nation’s wealth"

pacer$ideoFairShareWealth <- droplevels(pacer$ideoFairShareWealth)
pacer_w2$ideoFairShareWealth <- droplevels(pacer_w2$ideoFairShareWealth)

levels(pacer$ideoFairShareWealth)[1:5] <- rev(levels(pacer$ideoFairShareWealth)[1:5])
levels(pacer_w2$ideoFairShareWealth)[1:5] <- rev(levels(pacer_w2$ideoFairShareWealth)[1:5])

levels(pacer$ideoFairShareWealth)[6]  <- levels(pacer_w2$ideoFairShareWealth)[6] <- "Don't know"

names(bes)[grepl("lr3", names(bes))] <- gsub("lr3","ideoFairShareWealth",names(bes)[grepl("lr3", names(bes))])

# bes$lr4 and pacer$ideoOneLawRich

pacer$ideoOneLawRich <- droplevels(pacer$ideoOneLawRich)
pacer_w2$ideoOneLawRich <- droplevels(pacer_w2$ideoOneLawRich)
levels(pacer$ideoOneLawRich)[6]  <- levels(pacer_w2$ideoOneLawRich)[6] <- "Don't know"

names(bes)[grepl("lr4", names(bes))] <- gsub("lr4","ideoOneLawRich",names(bes)[grepl("lr4", names(bes))])

# benefits

pacer$reasonForUnemployment <- droplevels(pacer$reasonForUnemployment)
pacer_w2$reasonForUnemployment <- droplevels(pacer_w2$reasonForUnemployment)

pacer$immigrantsWelfareState <- droplevels(pacer$immigrantsWelfareState)
pacer_w2$immigrantsWelfareState <- droplevels(pacer_w2$immigrantsWelfareState)

pacer$govtHandouts <- droplevels(pacer$govtHandouts)
pacer_w2$govtHandouts <- droplevels(pacer_w2$govtHandouts)

pacer$benefitsNotDeserved <- droplevels(pacer$benefitsNotDeserved)
pacer_w2$benefitsNotDeserved <- droplevels(pacer_w2$benefitsNotDeserved)

# economic risks

pacer$riskPoverty <- droplevels(pacer$riskPoverty)
pacer_w2$riskPoverty <- droplevels(pacer_w2$riskPoverty)

pacer$riskUnemployment <- droplevels(pacer$riskUnemployment)
pacer_w2$riskUnemployment <- droplevels(pacer_w2$riskUnemployment)

# lockdownApproval

pacer$lockdownApproval <- droplevels(pacer$lockdownApproval)
pacer_w2$lockdownApproval <- droplevels(pacer_w2$lockdownApproval)

levels(pacer$lockdownApproval)[length(levels(pacer$lockdownApproval))] <- "Don't know"
levels(pacer_w2$lockdownApproval)[length(levels(pacer_w2$lockdownApproval))] <- "Don't know"

# workingStatus

workingStatus_levels <- c(levels(pacer$workingStatus), "Skipped", "Not Asked")

pacer$workingStatus <- factor(pacer$workingStatus, levels = workingStatus_levels)
pacer_w2$workingStatus <- factor(pacer_w2$workingStatus, levels = workingStatus_levels)

levels(bes$workingStatusW1W2W3W4W5)[!levels(bes$workingStatusW1W2W3W4W5) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW1W2W3W4W5 <- factor(bes$workingStatusW1W2W3W4W5, levels = workingStatus_levels)

levels(bes$workingStatusW6_W12)[!levels(bes$workingStatusW6_W12) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW6_W12 <- factor(bes$workingStatusW6_W12, levels = workingStatus_levels)

bes$workingStatusW16W17W18 <- factor(bes$workingStatusW16W17W18, labels= c(workingStatus_levels[1:9])) # Missing labels on this variable in the BES, for some reason
bes$workingStatusW16W17W18 <- factor(bes$workingStatusW16W17W18, levels = workingStatus_levels)

levels(bes$workingStatusW14W15)[!levels(bes$workingStatusW14W15) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW14W15 <- factor(bes$workingStatusW14W15, levels = workingStatus_levels)

levels(bes$workingStatusW19)[!levels(bes$workingStatusW19) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW19 <- factor(bes$workingStatusW19, levels = workingStatus_levels)

levels(bes$workingStatusW20)[!levels(bes$workingStatusW20) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW20 <- factor(bes$workingStatusW20, levels = workingStatus_levels)

levels(bes$workingStatusW21)[!levels(bes$workingStatusW21) %in% workingStatus_levels] <- "Working part time (8-29 hours a week)"
bes$workingStatusW21 <- factor(bes$workingStatusW21, levels = workingStatus_levels)

# incomeChangeHH 

pacer$income_decrease_binary <- pacer$incomeChangeHH %in% c("It has decreased by less than a quarter", "It has decreased by between a quarter and a half", "It is less than half what it was")

pacer_w2$income_decrease_binary <- pacer_w2$incomeChangeHH %in% c("It has decreased by less than a quarter", "It has decreased by between a quarter and a half", "It is less than half what it was")

bes$incomeChangeHHW20 <- bes$hhIncomeCoronaW20

levels(bes$incomeChangeHHW20)[levels(bes$incomeChangeHHW20) == "It is less than half of what it was"] <- "It is less than half what it was"
levels(bes$incomeChangeHHW20)[levels(bes$incomeChangeHHW20) == "Don't know"] <- "Don’t know"
bes$incomeChangeHHW20 <- droplevels(bes$incomeChangeHHW20)
bes$incomeChangeHHW20 <- factor(bes$incomeChangeHHW20, levels = levels(pacer_w2$incomeChangeHH))

bes$income_decrease_binaryW20 <- bes$incomeChangeHHW20 %in% c("It has decreased by less than a quarter", "It has decreased by between a quarter and a half", "It is less than half what it was")
bes$income_decrease_binaryW20[is.na(bes$incomeChangeHHW20)] <- NA

# income_guarantee_binary = 1 if
    # Coronavirus Job Retention Scheme (for furloughed employees of some companies)
    # Coronavirus Self-Employment Income Support Scheme
    # Universal credit
    # Other state benefits

pacer$income_guarantee_binary <- pacer$incomeGuaranteePersonal %in% levels(pacer$incomeGuaranteePersonal)[1:4] | pacer$incomeGuaranteeHH %in% levels(pacer$incomeGuaranteePersonal)[1:4]

pacer_w2$income_guarantee_binary <- pacer_w2$incomeGuaranteePersonal %in% levels(pacer_w2$incomeGuaranteePersonal)[1:4] | pacer_w2$incomeGuaranteeHH %in% levels(pacer_w2$incomeGuaranteePersonal)[1:4]

bes$income_guarantee_binaryW20 <- bes$incomeGuaranteeHH_1W20 == "Yes" | bes$incomeGuaranteeHH_2W20 == "Yes" | bes$incomeGuaranteeHH_3W20 == "Yes" |bes$incomeGuaranteeHH_4W20 == "Yes" | bes$incomeGuaranteePersonal_1W20 == "Yes" | bes$incomeGuaranteePersonal_2W20 == "Yes" | bes$incomeGuaranteePersonal_3W20 == "Yes" |bes$incomeGuaranteePersonal_4W20 == "Yes" 

# income_guarantee_covid_binary = 1 if
    # Coronavirus Job Retention Scheme (for furloughed employees of some companies)
    # Coronavirus Self-Employment Income Support Scheme

pacer$income_guarantee_covid_binary <- pacer$incomeGuaranteePersonal %in% levels(pacer$incomeGuaranteePersonal)[1:2] | pacer$incomeGuaranteeHH %in% levels(pacer$incomeGuaranteePersonal)[1:2]

pacer_w2$income_guarantee_covid_binary <- pacer_w2$incomeGuaranteePersonal %in% levels(pacer_w2$incomeGuaranteePersonal)[1:2] | pacer_w2$incomeGuaranteeHH %in% levels(pacer_w2$incomeGuaranteePersonal)[1:2]

bes$income_guarantee_covid_binaryW20 <- bes$incomeGuaranteeHH_1W20 == "Yes" | bes$incomeGuaranteeHH_2W20 == "Yes" | bes$incomeGuaranteePersonal_1W20 == "Yes" | bes$incomeGuaranteePersonal_2W20 == "Yes" 

# income_guarantee_other_binary = 1 if
    # Universal credit
    # Other state benefits

pacer$income_guarantee_other_binary <- pacer$income_guarantee_binary & !pacer$income_guarantee_covid_binary
pacer_w2$income_guarantee_other_binary <- pacer_w2$income_guarantee_binary & !pacer_w2$income_guarantee_covid_binary
bes$income_guarantee_other_binaryW20 <- bes$income_guarantee_binaryW20 & !bes$income_guarantee_covid_binaryW20

# income_guarantee_covid_job_binary = 1 if
      # Coronavirus Job Retention Scheme (for furloughed employees of some companies)

pacer$income_guarantee_covid_job_binary <- pacer$incomeGuaranteePersonal == "Coronavirus Job Retention Scheme (for furloughed employees of some companies)" | pacer$incomeGuaranteeHH == "Coronavirus Job Retention Scheme (for furloughed employees of some companies)"

pacer_w2$income_guarantee_covid_job_binary <- pacer_w2$incomeGuaranteePersonal == "Coronavirus Job Retention Scheme (for furloughed employees of some companies)" | pacer_w2$incomeGuaranteeHH == "Coronavirus Job Retention Scheme (for furloughed employees of some companies)"

bes$income_guarantee_covid_job_binaryW20 <- bes$incomeGuaranteeHH_1W20 == "Yes" | bes$incomeGuaranteePersonal_1W20 == "Yes" 

# income_guarantee_covid_self_binary = 1 if
# Coronavirus Job Retention Scheme (for furloughed employees of some companies)

pacer$income_guarantee_covid_self_binary <- pacer$incomeGuaranteePersonal == "Coronavirus Self-Employment Income Support Scheme" | pacer$incomeGuaranteeHH == "Coronavirus Self-Employment Income Support Scheme"

pacer_w2$income_guarantee_covid_self_binary <- pacer_w2$incomeGuaranteePersonal == "Coronavirus Self-Employment Income Support Scheme" | pacer_w2$incomeGuaranteeHH == "Coronavirus Self-Employment Income Support Scheme"

bes$income_guarantee_covid_self_binaryW20 <- bes$incomeGuaranteeHH_2W20 == "Yes" | bes$incomeGuaranteePersonal_2W20 == "Yes" 

# Symptoms

symptoms <- c("No Covid symptoms", "Could still do all daily activities", "Had to stop some daily activities", "Most daily activities were not possible", "Required hospitalisation", "Required ventilation", "They died", "Don't know")

pacer$severity_self <- ifelse(is.na(pacer$cvSeveritySelf), "No Covid symptoms", as.character(pacer$cvSeveritySelf))

pacer$severity_self <- factor(pacer$severity_self, levels = symptoms)

pacer$severity_family <- ifelse(is.na(pacer$cvSeverityFamily), "No Covid symptoms", as.character(pacer$cvSeverityFamily))

pacer$severity_family <- factor(pacer$severity_family, levels = symptoms)

pacer$severity_other <- ifelse(is.na(pacer$cvSeverityOth), "No Covid symptoms", as.character(pacer$cvSeverityOth))

pacer$severity_other <- factor(pacer$severity_other, levels = symptoms)

pacer$cv_suspect_any <- apply(pacer[,c("CVsuspect_1", "CVsuspect_2", "CVsuspect_3")],1,function(x) any(x == "Yes"))


pacer_w2$severity_self <- ifelse(is.na(pacer_w2$cvSeveritySelf), "No Covid symptoms", as.character(pacer_w2$cvSeveritySelf))

pacer_w2$severity_self <- factor(pacer_w2$severity_self, levels = symptoms)

pacer_w2$severity_family <- ifelse(is.na(pacer_w2$cvSeverityFamily), "No Covid symptoms", as.character(pacer_w2$cvSeverityFamily))

pacer_w2$severity_family <- factor(pacer_w2$severity_family, levels = symptoms)

pacer_w2$severity_other <- ifelse(is.na(pacer_w2$cvSeverityOth), "No Covid symptoms", as.character(pacer_w2$cvSeverityOth))

pacer_w2$severity_other <- factor(pacer_w2$severity_other, levels = symptoms)

pacer_w2$cv_suspect_any <- apply(pacer_w2[,c("CVsuspect_1", "CVsuspect_2", "CVsuspect_3")],1,function(x) any(x == "Yes"))


bes$severity_selfW20 <- ifelse(is.na(bes$cvSeveritySelfW20), "No Covid symptoms", as.character(bes$cvSeveritySelfW20))
bes$severity_selfW20[bes$wave20 != 1] <- NA
bes$severity_selfW20 <- factor(bes$severity_selfW20, levels = symptoms)

bes$severity_selfW21 <- ifelse(is.na(bes$cvSeveritySelfW21), "No Covid symptoms", as.character(bes$cvSeveritySelfW21))
bes$severity_selfW21[bes$wave21 != 1] <- NA
bes$severity_selfW21 <- factor(bes$severity_selfW21, levels = symptoms)

bes$severity_familyW20 <- ifelse(is.na(bes$cvSeverityFamilyW20), "No Covid symptoms", as.character(bes$cvSeverityFamilyW20))
bes$severity_familyW20[bes$wave20 != 1] <- NA
bes$severity_familyW20[which(bes$severity_familyW20 == "They passed away")] <- "They died"
bes$severity_familyW20 <- factor(bes$severity_familyW20, levels = symptoms)

bes$severity_familyW21 <- ifelse(is.na(bes$cvSeverityFamilyW21), "No Covid symptoms", as.character(bes$cvSeverityFamilyW21))
bes$severity_familyW21[bes$wave21 != 1] <- NA
bes$severity_familyW21[which(bes$severity_familyW21 == "They passed away")] <- "They died"
bes$severity_familyW21 <- factor(bes$severity_familyW21, levels = symptoms)

bes$severity_otherW20 <- ifelse(is.na(bes$cvSeverityCloseW20), "No Covid symptoms", as.character(bes$cvSeverityCloseW20))
bes$severity_otherW20[bes$wave20 != 1] <- NA
bes$severity_otherW20[which(bes$severity_otherW20 == "They passed away")] <- "They died"
bes$severity_otherW20 <- factor(bes$severity_otherW20, levels = symptoms)

bes$severity_otherW21 <- ifelse(is.na(bes$cvSeverityCloseW21), "No Covid symptoms", as.character(bes$cvSeverityCloseW21))
bes$severity_otherW21[bes$wave21 != 1] <- NA
bes$severity_otherW21[which(bes$severity_otherW21 == "They passed away")] <- "They died"
bes$severity_otherW21 <- factor(bes$severity_otherW21, levels = symptoms)

bes$cv_suspect_anyW20 <- !(bes$severity_selfW20 == "No Covid symptoms" & bes$severity_familyW20 == "No Covid symptoms" & bes$severity_otherW20 == "No Covid symptoms")

bes$cv_suspect_anyW21 <- !(bes$severity_selfW21 == "No Covid symptoms" & bes$severity_familyW21 == "No Covid symptoms" & bes$severity_otherW21 == "No Covid symptoms")

## Weight variables

bes$cross_sectional_weight_W1 <- bes$wt_full_W1
bes$cross_sectional_weight_W2 <- bes$wt_full_W2
bes$cross_sectional_weight_W3 <- bes$wt_full_W3
bes$cross_sectional_weight_W4 <- bes$wt_full_W4
bes$cross_sectional_weight_W5 <- bes$wt_full_W5
bes$cross_sectional_weight_W6 <- bes$wt_new_W6
bes$cross_sectional_weight_W7 <- bes$wt_new_W7
bes$cross_sectional_weight_W8 <- bes$wt_new_W8
bes$cross_sectional_weight_W9 <- bes$wt_new_W9
bes$cross_sectional_weight_W10 <- bes$wt_new_W10
bes$cross_sectional_weight_W11 <- bes$wt_new_W11
bes$cross_sectional_weight_W12 <- bes$wt_new_W12
bes$cross_sectional_weight_W13 <- bes$wt_new_W13_result
bes$cross_sectional_weight_W14 <- bes$wt_new_W14
bes$cross_sectional_weight_W15 <- bes$wt_new_W15
bes$cross_sectional_weight_W16 <- bes$wt_new_W16
bes$cross_sectional_weight_W17 <- bes$wt_new_W17
bes$cross_sectional_weight_W18 <- bes$wt_new_W18
bes$cross_sectional_weight_W19 <- bes$wt_new_W19_result
bes$cross_sectional_weight_W20 <- bes$wt_new_W20
bes$cross_sectional_weight_W21 <- bes$wt_new_W21

pacer$cross_sectional_weight_ <- pacer$weight
pacer_w2$cross_sectional_weight_ <- pacer_w2$weight

## ##########
## Add pacer variables to BES

vars_to_add <- c("redistSelf", "ptvCon", "ptvLab", "ptvLD", "taxSpendSelf" ,"trustWestminster" ,"deficitReduce", "riskTaking", "reasonForUnemployment" , "immigrantsWelfareState", "approveUKGovt", "govtHandleCostLive", "govtHandleEcon", "govtHandleImmig", "govtHandleNHS", "govtHandleEduc", "govtHandleLevelCrime", "govtCorpSupport", "ideoBHPSPprivateEnterprise", "ideoBHPSSstateOwnership", "ideoBHPSJjobForAll", "ideoNoNeedUnions", "ideoFairShareWealth", "ideoOneLawRich", "govtHandouts", "benefitsNotDeserved", "riskPoverty", "riskUnemployment", "lockdownApproval", "workingStatus", "furloughedPersonal", "furloughedHH", "incomeChangeHH", "govtEconProtectHH",  "income_decrease_binary", "incomeGuaranteePersonal", "incomeGuaranteeHH", "income_guarantee_binary", "income_guarantee_covid_binary", "income_guarantee_covid_self_binary", "income_guarantee_covid_job_binary", "income_guarantee_other_binary", "cv_suspect_any", "severity_self", "severity_family", "severity_other", "polAttention", "cross_sectional_weight_")

bes$wave19.5 <- as.numeric(bes$id %in% pacer$id)
bes$wave20.5 <- as.numeric(bes$id %in% pacer_w2$id)

for(v in vars_to_add){
  
  bes[[paste0(v,"W19.5")]] <- pacer[[v]][match(bes$id, pacer$id)]
  bes[[paste0(v,"W20.5")]] <- pacer_w2[[v]][match(bes$id, pacer_w2$id)]
  
}

## Reformat BES to long format
bes$wave19.5 <- as.integer(bes$wave19.5)
bes$wave20.5 <- as.integer(bes$wave20.5)


bes_long_base <- melt.data.table(data.table(bes[,c("id", names(bes)[grep("^wave\\d+", names(bes))])]), id.vars = "id")
bes_long_base$wave <- as.numeric(gsub("wave","",bes_long_base$variable))
bes_long_base$variable <- NULL
bes_long_base$id_in_wave <- bes_long_base$value
bes_long_base$value <- NULL

# Specify variables for melting

bes_long <- bes_long_base

bes$workingStatusPartnerW20 <- NULL
bes$incomeGuaranteePersonal_1W20 <- NULL
bes$incomeGuaranteePersonal_2W20 <- NULL
bes$incomeGuaranteePersonal_3W20 <- NULL
bes$incomeGuaranteePersonal_4W20 <- NULL
bes$incomeGuaranteePersonal_noneW20 <- NULL
bes$incomeGuaranteeHH_1W20 <- NULL
bes$incomeGuaranteeHH_2W20 <- NULL
bes$incomeGuaranteeHH_3W20 <- NULL
bes$incomeGuaranteeHH_4W20 <- NULL
bes$incomeGuaranteeHH_noneW20 <- NULL

bes$ptvCon_imageW21 <- NULL
bes$ptvLab_imageW21 <- NULL

var_name <- "ideoOneLawRich"
var_name <- vars_to_add[8]

# Rename variables to old BES format (W1W2W3W4W5W6 not W1_W6)

bes$riskTakingW1 <- bes$riskTakingW1_W5
bes$riskTakingW2 <- bes$riskTakingW1_W5
bes$riskTakingW3 <- bes$riskTakingW1_W5
bes$riskTakingW4 <- bes$riskTakingW1_W5
bes$riskTakingW5 <- bes$riskTakingW1_W5
bes$riskTakingW1_W5 <- NULL

bes$ideoFairShareWealthW1 <- bes$ideoFairShareWealthW1_W5
bes$ideoFairShareWealthW2 <- bes$ideoFairShareWealthW1_W5
bes$ideoFairShareWealthW3 <- bes$ideoFairShareWealthW1_W5
bes$ideoFairShareWealthW4 <- bes$ideoFairShareWealthW1_W5
bes$ideoFairShareWealthW5 <- bes$ideoFairShareWealthW1_W5
bes$ideoFairShareWealthW1_W5 <- NULL

bes$ideoFairShareWealthW7 <- bes$ideoFairShareWealthW7_W9
bes$ideoFairShareWealthW8 <- bes$ideoFairShareWealthW7_W9
bes$ideoFairShareWealthW9 <- bes$ideoFairShareWealthW7_W9
bes$ideoFairShareWealthW7_W9 <- NULL

bes$ideoFairShareWealthW10 <- bes$ideoFairShareWealthW10_W12
bes$ideoFairShareWealthW11 <- bes$ideoFairShareWealthW10_W12
bes$ideoFairShareWealthW12 <- bes$ideoFairShareWealthW10_W12
bes$ideoFairShareWealthW10_W12 <- NULL

bes$ideoFairShareWealthW14 <- bes$ideoFairShareWealthW14W15
bes$ideoFairShareWealthW15 <- bes$ideoFairShareWealthW14W15
bes$ideoFairShareWealthW14W15 <- NULL

bes$ideoOneLawRichW1 <- bes$ideoOneLawRichW1_W5
bes$ideoOneLawRichW2 <- bes$ideoOneLawRichW1_W5
bes$ideoOneLawRichW3 <- bes$ideoOneLawRichW1_W5
bes$ideoOneLawRichW4 <- bes$ideoOneLawRichW1_W5
bes$ideoOneLawRichW5 <- bes$ideoOneLawRichW1_W5
bes$ideoOneLawRichW1_W5 <- NULL

bes$ideoOneLawRichW7 <- bes$ideoOneLawRichW7_W9
bes$ideoOneLawRichW8 <- bes$ideoOneLawRichW7_W9
bes$ideoOneLawRichW9 <- bes$ideoOneLawRichW7_W9
bes$ideoOneLawRichW7_W9 <- NULL

bes$ideoOneLawRichW10 <- bes$ideoOneLawRichW10_W12
bes$ideoOneLawRichW11 <- bes$ideoOneLawRichW10_W12
bes$ideoOneLawRichW12 <- bes$ideoOneLawRichW10_W12
bes$ideoOneLawRichW10_W12 <- NULL

bes$ideoOneLawRichW14 <- bes$ideoOneLawRichW14W15
bes$ideoOneLawRichW15 <- bes$ideoOneLawRichW14W15
bes$ideoOneLawRichW14W15 <- NULL

bes$workingStatusW1 <- bes$workingStatusW1W2W3W4W5
bes$workingStatusW2 <- bes$workingStatusW1W2W3W4W5
bes$workingStatusW3 <- bes$workingStatusW1W2W3W4W5
bes$workingStatusW4 <- bes$workingStatusW1W2W3W4W5
bes$workingStatusW5 <- bes$workingStatusW1W2W3W4W5
bes$workingStatusW1W2W3W4W5 <- NULL

bes$workingStatusW6 <- bes$workingStatusW6_W12
bes$workingStatusW7 <- bes$workingStatusW6_W12
bes$workingStatusW8 <- bes$workingStatusW6_W12
bes$workingStatusW9 <- bes$workingStatusW6_W12
bes$workingStatusW10 <- bes$workingStatusW6_W12
bes$workingStatusW11 <- bes$workingStatusW6_W12
bes$workingStatusW12 <- bes$workingStatusW6_W12
bes$workingStatusW6_W12 <- NULL

bes$workingStatusW14 <- bes$workingStatusW14W15
bes$workingStatusW15 <- bes$workingStatusW14W15
bes$workingStatusW14W15 <- NULL

bes$workingStatusW16 <- bes$workingStatusW16W17W18
bes$workingStatusW17 <- bes$workingStatusW16W17W18
bes$workingStatusW18 <- bes$workingStatusW16W17W18

bes$workingStatusW16W17W18 <- NULL

bes$ptvConW14 <- bes$ptvConExpW14
bes$ptvLabW14 <- bes$ptvLabExpW14
bes$ptvLDW14 <- bes$ptvLDExpW14

bes$ptvConExpW14 <- NULL
bes$ptvLabExpW14 <- NULL
bes$ptvLDExpW14 <- NULL

bes$polAttentionpdlW12 <- NULL 


bes_tmp <- as_tibble(bes)

# options(warn = 2)  
out <- lapply(vars_to_add, function(x){
  tmp <- bes_tmp %>%
    select(c("id", starts_with(x))) %>%
    pivot_longer(cols = c(starts_with(x)),
                 names_to = c("variable","wave"),
                 names_sep = paste0(x,"W")) %>%
    mutate(var = x) %>%
    select(id, wave, value) 
  names(tmp)[3] <- x
  tmp
})

out_joined <- out %>% 
  reduce(full_join, by = c("id","wave"))

bes_long_all <- as_tibble(bes_long) %>% 
  mutate(wave = as.character(wave)) %>%
  full_join(out_joined, by = c("id", "wave"))
  
bes_long_all <- data.frame(bes_long_all)
bes_long_all[bes_long_all$id_in_wave == 0,-c(1,2,3)] <- NA
bes_long <- data.table(bes_long_all)

## ###############
## DV factor recodes (reorder and drop DKs)
## All "ideological" DVs recoded such that leftist views are high numbers and rightist views are low numbers

# redistSelf

bes_long$redistSelf_dk <- bes_long$redistSelf == "Don't know"
bes_long$redistSelf <- factor(bes_long$redistSelf, levels = rev(c("Government should try to make incomes equal", 1:9, "Government should be less concerned about equal incomes")))

# ptvXXX

bes_long$ptvCon_dk <- bes_long$ptvCon == "Don't know"
bes_long$ptvLab_dk <- bes_long$ptvLab == "Don't know"
bes_long$ptvLD_dk <- bes_long$ptvLD == "Don't know"

bes_long$ptvCon <- factor(bes_long$ptvCon, levels = c("Very unlikely", 1:9, "Very likely"))
bes_long$ptvLab <- factor(bes_long$ptvLab, levels = c("Very unlikely", 1:9, "Very likely"))
bes_long$ptvLD <- factor(bes_long$ptvLD, levels = c("Very unlikely", 1:9, "Very likely"))

# taxSpendSelf

bes_long$taxSpendSelf_dk <- bes_long$taxSpendSelf == "Don't know"

bes_long$taxSpendSelf <- factor(bes_long$taxSpendSelf, levels = c("Government should cut taxes a lot and spend much less on health and social services", 1:9, "Government should increase taxes a lot and spend much more on health and social services"))

# trustWestminster

bes_long$trustWestminster <- factor(bes_long$trustWestminster, levels = c("Hardly ever", "Some of the time", "Most of the time", "Just about always"))

# deficitReduce

bes_long$deficitReduce <- factor(bes_long$deficitReduce, levels = rev(c("It is completely unnecessary", "It is not necessary but it would be desirable", "It is important but not absolutely  necessary", "It is completely necessary")))

# riskTaking

bes_long$riskTaking <- factor(bes_long$riskTaking, levels = c("Very unwilling to take risks", "Somewhat unwilling to take risks", "Somewhat willing to take risks", "Very willing to take risks"))

# reasonForUnemployment

bes_long$reasonForUnemployment <- factor(bes_long$reasonForUnemployment, levels = c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree"))

# immigrantsWelfareState

bes_long$immigrantsWelfareState <- factor(bes_long$immigrantsWelfareState, levels = rev(c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree")))

bes_long$benefitsNotDeserved <- factor(bes_long$benefitsNotDeserved, levels = rev(c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree")))

bes_long$govtHandouts <- factor(bes_long$govtHandouts, levels = rev(c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree")))

# approveUKGovt

bes_long$approveUKGovt <- factor(bes_long$approveUKGovt, levels = c("Strongly disapprove", "Disapprove", "Neither approve nor disapprove", "Approve", "Strongly approve"))

# govtHandle...

bes_long$govtHandleCostLive <- factor(bes_long$govtHandleCostLive, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))
bes_long$govtHandleEcon <- factor(bes_long$govtHandleEcon, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))
bes_long$govtHandleImmig <- factor(bes_long$govtHandleImmig, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))
bes_long$govtHandleNHS <- factor(bes_long$govtHandleNHS, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))
bes_long$govtHandleEduc <- factor(bes_long$govtHandleEduc, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))
bes_long$govtHandleLevelCrime <- factor(bes_long$govtHandleLevelCrime, levels = c("Very badly", "Fairly badly", "Neither well nor badly", "Fairly well", "Very well"))

# govtCorpSupport

bes_long$govtCorpSupport <- factor(bes_long$govtCorpSupport, levels = c("0 - Government should leave large companies to deal with the crisis on their own", 1:9, "10 - Government should provide all the financial support large companies need to survive the crisis"))

# ideoBHPS...

bes_long$ideoBHPSPprivateEnterprise <- factor(bes_long$ideoBHPSPprivateEnterprise, levels = rev(c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree")))
bes_long$ideoBHPSSstateOwnership <- factor(bes_long$ideoBHPSSstateOwnership, levels = c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree"))
bes_long$ideoBHPSJjobForAll <- factor(bes_long$ideoBHPSJjobForAll, levels = c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree"))
bes_long$ideoNoNeedUnions <- factor(bes_long$ideoNoNeedUnions, levels = rev(c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree")))
## BES wording: "Ordinary working people do not get their fair share of the nation’s wealth"
bes_long$ideoFairShareWealth <- factor(bes_long$ideoFairShareWealth, levels = c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree"))
bes_long$ideoOneLawRich <- factor(bes_long$ideoOneLawRich, levels = c("Strongly disagree", "Disagree", "Neither agree nor disagree", "Agree", "Strongly agree"))

# risk

bes_long$riskPoverty <- factor(bes_long$riskPoverty, levels = c("Very unlikely", "Fairly unlikely", "Neither likely nor unlikely", "Fairly likely", "Very likely"))

bes_long$riskUnemployment <- factor(bes_long$riskUnemployment, levels = c("Very unlikely", "Fairly unlikely", "Neither likely nor unlikely", "Fairly likely", "Very likely"))

#  lockdownApproval

bes_long$lockdownApproval <- factor(bes_long$lockdownApproval, levels = c("0 - Strongly oppose lockdown", 1:9, "10 - Strongly support lockdown"))

# govtEconProtectHH
bes_long$govtEconProtectHH <- factor(bes_long$govtEconProtectHH, levels = c("0 - Household has received no government financial help", 1:9, "10 - Household has received a lot of government financial help"))

## Convenience variables

bes_long$id_in_pacer_w1 <- bes_long$id %in% bes_long[wave == 19.5 & id_in_wave == 1]$id

bes_long$id_in_pacer_w2 <- bes_long$id %in% bes_long[wave == 20.5 & id_in_wave == 1]$id

bes_long$id_in_bes_w20 <- bes_long$id %in% bes_long[wave == 20 & id_in_wave == 1]$id

bes_long$id_in_bes_w21 <- bes_long$id %in% bes_long[wave == 21 & id_in_wave == 1]$id

bes_long$id_in_any_pacer <- bes_long$id %in% unique(bes_long[wave %in% c(19.5, 20.5) & id_in_wave == 1]$id)

## Non-time-varying variables

bes_long$p_past_vote_2005 <- bes$p_past_vote_2005[match(bes_long$id, bes$id)]
bes_long$p_past_vote_2010 <- bes$p_past_vote_2010[match(bes_long$id, bes$id)]
bes_long$p_past_vote_2015 <- bes$p_past_vote_2015[match(bes_long$id, bes$id)]
bes_long$p_past_vote_2017 <- bes$p_past_vote_2017[match(bes_long$id, bes$id)]
bes_long$p_past_vote_2019 <- bes$p_past_vote_2019[match(bes_long$id, bes$id)]

bes_long$wave <- as.numeric(bes_long$wave)

covars <- tibble(bes) %>%
  transmute(id = id, 
            p_sexuality = coalesce(!!! select(., starts_with("p_sexuality"))),
            p_ethnicity = coalesce(!!! select(., starts_with("p_ethnicity"))),
            p_religion = coalesce(!!! select(., starts_with("p_religion"))),
            p_education = coalesce(!!! select(., starts_with("p_education"))),
            p_edlevel = coalesce(!!! select(., starts_with("p_edlevel"))),
            p_parent = coalesce(!!! select(., starts_with("p_parent"))),
            p_socgrade = coalesce(!!! select(., starts_with("p_socgrade"))),
            p_marital = coalesce(!!! select(., starts_with("p_marital"))),
            p_housing = coalesce(!!! select(., starts_with("p_housing"))),
            p_housing = coalesce(!!! select(., starts_with("p_housing"))),
            p_gross_personal = coalesce(!!! select(., starts_with("p_gross_personal"))),
            p_gross_household = coalesce(!!! select(., starts_with("p_gross_household"))))

bes_long <- tibble(bes_long) %>% left_join(covars, by = "id") %>% data.table()

prior_gov_support <- tibble(bes) %>%
  summarise(id = id,
            gov_support_past = case_when(allSourceIncome_4W6 == "Yes" ~ "Yes",
                                          allSourceIncome_4W11 == "Yes" ~ "Yes",
                                          allSourceIncome_5W6 == "Yes" ~ "Yes",
                                          allSourceIncome_5W11 == "Yes" ~ "Yes",
                                          allSourceIncome_6W6 == "Yes" ~ "Yes",
                                          allSourceIncome_6W11 == "Yes" ~ "Yes",
                                          allSourceIncome_7W6 == "Yes" ~ "Yes",
                                          allSourceIncome_7W11 == "Yes" ~ "Yes",
                                          TRUE ~ "No"),
            gov_support_past = ifelse(wave6 ==0 & wave11 == 0, NA, gov_support_past))


bes_long$gov_support_past <- prior_gov_support$gov_support_past[match(bes_long$id, prior_gov_support$id)]

setkey(bes_long, id, wave)

save(bes_long, file = "../generated_data/bes_w1_w21_pacer_w1_w2_merged.Rdata")

